{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "python_files = [os.path.join(f[0], fname) for f in os.walk(\"/home/hadoop/Downloads/xformer-multi-source-domain-adaptation-master/\") for fname in f[2] if fname[-3:]=='.py']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dst_pys = [fname.replace(\"/home/hadoop/Downloads/xformer-multi-source-domain-adaptation-master/\", \"../../\") for fname in python_files]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for src, dst in zip(python_files, dst_pys):\n",
    "    os.system(\"mv %s %s\"%(src, dst))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "python_files = [os.path.join(f[0], fname) for f in os.walk(\"../../\") for fname in f[2] if fname[-3:]=='.py']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "flag = False\n",
    "print_lines = \"\"\n",
    "log_lines = \"\"\n",
    "for fname in python_files:\n",
    "    with open(fname, \"r\") as fr:\n",
    "        lines = [line for line in fr]\n",
    "    with open(fname, \"w\") as fw:\n",
    "        for line in lines:\n",
    "            if \"wandb.log({\" in line:\n",
    "                flag = False if \"})\" in line else True\n",
    "                print_lines = line.replace(\"wandb.log\", \"print\")\n",
    "                log_lines = line\n",
    "                if not flag:\n",
    "                    fw.write(log_lines)\n",
    "                    fw.write(print_lines)\n",
    "                    print_lines = \"\"\n",
    "            elif flag:\n",
    "                flag = False if \"})\" in line else True\n",
    "                print_lines = print_lines + line\n",
    "                log_lines = log_lines + line\n",
    "                if not flag:\n",
    "                    fw.write(log_lines)\n",
    "                    fw.write(print_lines)\n",
    "                    print_lines = \"\"\n",
    "            else:\n",
    "                fw.write(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "for fname in python_files:\n",
    "    with open(fname, \"r\") as fr:\n",
    "        lines = [line for line in fr]\n",
    "    with open(fname, \"w\") as fw:\n",
    "        for line in lines:\n",
    "            if \"wandb.run.summary\" in line:\n",
    "                line = line.replace(\"wandb.run.summary\", \n",
    "                                    \"fitlog.add_best_metric\").replace('[', \n",
    "                                                                      '({').replace(']', \n",
    "                                                                                   ':').replace('=', '')\n",
    "                line = line[:-1] + '})' + '\\n'\n",
    "            fw.write(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_loc = \"../../data/\" \n",
    "train_pct = 0.9 \n",
    "n_gpu = 1 \n",
    "n_epochs = 5 \n",
    "domains = \"books dvd electronics kitchen_&_housewares \"\n",
    "run_name = \"basic-distilbert-${2}\"\n",
    "batch_size = 8\n",
    "lr = 0.00003"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import gc\n",
    "import os\n",
    "import random\n",
    "from typing import AnyStr\n",
    "from typing import List\n",
    "import ipdb\n",
    "from collections import defaultdict\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "import fitlog\n",
    "from torch.optim.lr_scheduler import LambdaLR\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data import Subset\n",
    "from torch.utils.data import random_split\n",
    "from torch.optim import Adam\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../..\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AdamW\n",
    "from transformers import DistilBertConfig\n",
    "from transformers import DistilBertTokenizer\n",
    "from transformers import DistilBertModel\n",
    "from transformers import BertConfig\n",
    "from transformers import BertTokenizer\n",
    "from transformers import BertModel\n",
    "from transformers import get_linear_schedule_with_warmup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datareader import MultiDomainSentimentDataset\n",
    "from datareader import collate_batch_transformer\n",
    "from metrics import MultiDatasetClassificationEvaluator, acc_f1\n",
    "from metrics import ClassificationEvaluator\n",
    "\n",
    "from metrics import plot_label_distribution\n",
    "from model import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(\n",
    "        model: torch.nn.Module,\n",
    "        train_dls: List[DataLoader],\n",
    "        optimizer: torch.optim.Optimizer,\n",
    "        scheduler: LambdaLR,\n",
    "        validation_evaluator: MultiDatasetClassificationEvaluator,\n",
    "        n_epochs: int,\n",
    "        device: AnyStr,\n",
    "        log_interval: int = 1,\n",
    "        patience: int = 10,\n",
    "        model_dir: str = \"wandb_local\",\n",
    "        gradient_accumulation: int = 1,\n",
    "        domain_name: str = ''\n",
    "):\n",
    "    #best_loss = float('inf')\n",
    "    best_acc = 0.0\n",
    "    patience_counter = 0\n",
    "\n",
    "    epoch_counter = 0\n",
    "    total = sum(len(dl) for dl in train_dls)\n",
    "\n",
    "    # Main loop\n",
    "    while epoch_counter < n_epochs:\n",
    "        dl_iters = [iter(dl) for dl in train_dls]\n",
    "        dl_idx = list(range(len(dl_iters)))\n",
    "        finished = [0] * len(dl_iters)\n",
    "        i = 0\n",
    "        with tqdm(total=total, desc=\"Training\") as pbar:\n",
    "            while sum(finished) < len(dl_iters):\n",
    "                random.shuffle(dl_idx)\n",
    "                for d in dl_idx:\n",
    "                    domain_dl = dl_iters[d]\n",
    "                    batches = []\n",
    "                    try:\n",
    "                        for j in range(gradient_accumulation):\n",
    "                            batches.append(next(domain_dl))\n",
    "                    except StopIteration:\n",
    "                        finished[d] = 1\n",
    "                        if len(batches) == 0:\n",
    "                            continue\n",
    "                    optimizer.zero_grad()\n",
    "                    for batch in batches:\n",
    "                        model.train()\n",
    "                        batch = tuple(t.to(device) for t in batch)\n",
    "                        input_ids = batch[0]\n",
    "                        masks = batch[1]\n",
    "                        labels = batch[2]\n",
    "                        # Testing with random domains to see if any effect\n",
    "                        #domains = torch.tensor(np.random.randint(0, 16, batch[3].shape)).to(device)\n",
    "                        domains = batch[3]\n",
    "\n",
    "                        rst = model(input_ids, attention_mask=masks, domains=domains, labels=labels)
                        loss, logits = rst.loss, rst.logits\n",
    "                        loss = loss / gradient_accumulation\n",
    "\n",
    "                        if i % log_interval == 0:\n",
    "                            fitlog.add_metric({\n",
    "                                \"Loss\": loss.item()\n",
    "                            })\n",
    "\n",
    "                        loss.backward()\n",
    "                        i += 1\n",
    "                        pbar.update(1)\n",
    "\n",
    "                    optimizer.step()\n",
    "                    if scheduler is not None:\n",
    "                        scheduler.step()\n",
    "\n",
    "        gc.collect()\n",
    "\n",
    "        # Inline evaluation\n",
    "        (val_loss, acc, P, R, F1), _ = validation_evaluator.evaluate(model)\n",
    "        print(f\"Validation acc: {acc}\")\n",
    "\n",
    "        #torch.save(model.state_dict(), f'{model_dir}/model_{domain_name}.pth')\n",
    "\n",
    "        # Saving the best model and early stopping\n",
    "        #if val_loss < best_loss:\n",
    "        if acc > best_acc:\n",
    "            best_model = model.state_dict()\n",
    "            #best_loss = val_loss\n",
    "            best_acc = acc\n",
    "            #wandb.run.summary['best_validation_loss'] = best_loss\n",
    "            torch.save(model.state_dict(), f'{model_dir}/model_{domain_name}.pth')\n",
    "            patience_counter = 0\n",
    "            # Log to wandb\n",
    "            fitlog.add_metric({\n",
    "                'Validation accuracy': acc,\n",
    "                'Validation Precision': P,\n",
    "                'Validation Recall': R,\n",
    "                'Validation F1': F1,\n",
    "                'Validation loss': val_loss})\n",
    "        else:\n",
    "            patience_counter += 1\n",
    "            # Stop training once we have lost patience\n",
    "            if patience_counter == patience:\n",
    "                break\n",
    "\n",
    "        gc.collect()\n",
    "        epoch_counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class adict(dict):\n",
    "    ''' Attribute dictionary - a convenience data structure, similar to SimpleNamespace in python 3.3\n",
    "        One can use attributes to read/write dictionary content.\n",
    "    '''\n",
    "    def __init__(self, *av, **kav):\n",
    "        dict.__init__(self, *av, **kav)\n",
    "        self.__dict__ = self"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = adict(dataset_loc=\"data/\",\n",
    "            train_pct=0.9,\n",
    "            n_gpu=1,\n",
    "            n_epochs=5,\n",
    "            warmup_steps=200,\n",
    "            indices_dir=None,\n",
    "            domains=\"books dvd electronics kitchen_&_housewares\".split(\" \"),\n",
    "            tags=\"emnlp sentiment experiments\".split(\" \"),\n",
    "            run_name=\"basic-distilbert\",\n",
    "            model_dir=\"wandb_local/emnlp_sentiment_experiments\",\n",
    "            weight_decay=0.01,\n",
    "            full_bert=None,\n",
    "            seed=1000,\n",
    "            pretrained_model=None,\n",
    "            batch_size=8,\n",
    "            lr=0.00003)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a3f23a7495454929840388d32a3d4b35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "bert_model = 'bert-base-uncased'\n",
    "bert_config = BertConfig.from_pretrained(bert_model, num_labels=2)\n",
    "tokenizer = BertTokenizer.from_pretrained(bert_model)\n",
    "bert = BertForSequenceClassification.from_pretrained(bert_model, config=bert_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "bert_model = 'distilbert-base-uncased'\n",
    "bert_config = DistilBertConfig.from_pretrained(bert_model, num_labels=2)\n",
    "tokenizer = DistilBertTokenizer.from_pretrained(bert_model)\n",
    "bert = DistilBertForSequenceClassification.from_pretrained(bert_model, config=bert_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n",
      "wandb: Currently logged in as: marlonlu (use `wandb login --relogin` to force relogin)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                Tracking run with wandb version 0.10.22<br/>\n",
       "                Syncing run <strong style=\"color:#cdcd00\">basic-distilbert</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
       "                Project page: <a href=\"https://wandb.ai/marlonlu/domain-adaptation-sentiment-emnlp\" target=\"_blank\">https://wandb.ai/marlonlu/domain-adaptation-sentiment-emnlp</a><br/>\n",
       "                Run page: <a href=\"https://wandb.ai/marlonlu/domain-adaptation-sentiment-emnlp/runs/288w5p8h\" target=\"_blank\">https://wandb.ai/marlonlu/domain-adaptation-sentiment-emnlp/runs/288w5p8h</a><br/>\n",
       "                Run data is saved locally in <code>/home/hadoop/xformer-multi-source-domain-adaptation/emnlp_final_experiments/sentiment-analysis/wandb/run-20210322_210702-288w5p8h</code><br/><br/>\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h1>Run(288w5p8h)</h1><iframe src=\"https://wandb.ai/marlonlu/domain-adaptation-sentiment-emnlp/runs/288w5p8h\" style=\"border:none;width:100%;height:400px\"></iframe>"
      ],
      "text/plain": [
       "<wandb.sdk.wandb_run.Run at 0x7f3365023a58>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wandb.init(\n",
    "    project=\"domain-adaptation-sentiment-emnlp\",\n",
    "    name=args.run_name,\n",
    "    config={\n",
    "        \"epochs\": n_epochs,\n",
    "        \"learning_rate\": lr,\n",
    "        \"warmup\": args.warmup_steps,\n",
    "        \"weight_decay\": args.weight_decay,\n",
    "        \"batch_size\": batch_size,\n",
    "        \"train_split_percentage\": args.train_pct,\n",
    "        \"bert_model\": bert_model,\n",
    "        \"seed\": args.seed,\n",
    "        \"pretrained_model\": args.pretrained_model,\n",
    "        \"tags\": \",\".join(args.tags)\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "    if not os.path.exists(f\"{args.model_dir}/{Path(wandb.run.dir).name}\"):\n",
    "        os.makedirs(f\"{args.model_dir}/{Path(wandb.run.dir).name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_dsets = [MultiDomainSentimentDataset(\n",
    "    dataset_loc,\n",
    "    [domain],\n",
    "    tokenizer\n",
    ") for domain in domains.strip().split(\" \")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "    train_sizes = [int(len(dset) * args.train_pct) for j, dset in enumerate(all_dsets)]\n",
    "    val_sizes = [len(all_dsets[j]) - train_sizes[j] for j in range(len(train_sizes))]\n",
    "\n",
    "    accs = []\n",
    "    Ps = []\n",
    "    Rs = []\n",
    "    F1s = []\n",
    "    # Store labels and logits for individual splits for micro F1\n",
    "    labels_all = []\n",
    "    logits_all = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "        domain = args.domains[i]\n",
    "        test_dset = all_dsets[i]\n",
    "        # Override the domain IDs\n",
    "        k = 0\n",
    "        for j in range(len(all_dsets)):\n",
    "            if j != i:\n",
    "                all_dsets[j].set_domain_id(k)\n",
    "                k += 1\n",
    "        test_dset.set_domain_id(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "        if args.indices_dir is None:\n",
    "            subsets = [random_split(all_dsets[j], [train_sizes[j], val_sizes[j]])\n",
    "                       for j in range(len(all_dsets)) if j != i]\n",
    "            # Save the indices\n",
    "            with open(f'{args.model_dir}/train_idx_{domain}.txt', 'wt') as f, \\\n",
    "                    open(f'{args.model_dir}/val_idx_{domain}.txt', 'wt') as g:\n",
    "                for j, subset in enumerate(subsets):\n",
    "                    for idx in subset[0].indices:\n",
    "                        f.write(f'{j},{idx}\\n')\n",
    "                    for idx in subset[1].indices:\n",
    "                        g.write(f'{j},{idx}\\n')\n",
    "        else:\n",
    "            # load the indices\n",
    "            dset_choices = [all_dsets[j] for j in range(len(all_dsets)) if j != i]\n",
    "            subset_indices = defaultdict(lambda: [[], []])\n",
    "            with open(f'{args.indices_dir}/train_idx_{domain}.txt') as f, \\\n",
    "                    open(f'{args.indices_dir}/val_idx_{domain}.txt') as g:\n",
    "                for l in f:\n",
    "                    vals = l.strip().split(',')\n",
    "                    subset_indices[int(vals[0])][0].append(int(vals[1]))\n",
    "                for l in g:\n",
    "                    vals = l.strip().split(',')\n",
    "                    subset_indices[int(vals[0])][1].append(int(vals[1]))\n",
    "            subsets = [[Subset(dset_choices[d], subset_indices[d][0]), Subset(dset_choices[d], subset_indices[d][1])]\n",
    "                       for d in subset_indices]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "        device = torch.device(\"cuda:0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "        train_dls = [DataLoader(\n",
    "            subset[0],\n",
    "            batch_size=batch_size,\n",
    "            shuffle=True,\n",
    "            collate_fn=collate_batch_transformer\n",
    "        ) for subset in subsets]\n",
    "\n",
    "        val_ds = [subset[1] for subset in subsets]\n",
    "        validation_evaluator = MultiDatasetClassificationEvaluator(val_ds, device)\n",
    "\n",
    "\n",
    "        # Create the model\n",
    "        if args.full_bert:\n",
    "            bert = BertForSequenceClassification.from_pretrained(bert_model, config=bert_config).to(device)\n",
    "        else:\n",
    "            bert = DistilBertForSequenceClassification.from_pretrained(bert_model, config=bert_config).to(device)\n",
    "        model = VanillaBert(bert).to(device)\n",
    "        if args.pretrained_model is not None:\n",
    "            model.load_state_dict(torch.load(f\"{args.pretrained_model}/model_{domain}.pth\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "        batch_size = args.batch_size\n",
    "        lr = args.lr\n",
    "        weight_decay = args.weight_decay\n",
    "        n_epochs = args.n_epochs\n",
    "        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n",
    "        optimizer_grouped_parameters = [\n",
    "            {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n",
    "             'weight_decay': weight_decay},\n",
    "            {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
    "        ]\n",
    "        # optimizer = Adam(optimizer_grouped_parameters, lr=1e-3)\n",
    "        # scheduler = None\n",
    "        optimizer = AdamW(optimizer_grouped_parameters, lr=lr)\n",
    "        scheduler = get_linear_schedule_with_warmup(\n",
    "            optimizer,\n",
    "            args.warmup_steps,\n",
    "            n_epochs * sum([len(train_dl) for train_dl in train_dls])\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "    best_acc = 0.0\n",
    "    patience_counter = 0\n",
    "\n",
    "    epoch_counter = 0\n",
    "    total = sum(len(dl) for dl in train_dls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "        dl_iters = [iter(dl) for dl in train_dls]\n",
    "        dl_idx = list(range(len(dl_iters)))\n",
    "        finished = [0] * len(dl_iters)\n",
    "        i = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([<torch.utils.data.dataloader._SingleProcessDataLoaderIter at 0x7f33696d6550>,\n",
       "  <torch.utils.data.dataloader._SingleProcessDataLoaderIter at 0x7f336a080cc0>,\n",
       "  <torch.utils.data.dataloader._SingleProcessDataLoaderIter at 0x7f3368eacba8>],\n",
       " [0, 1, 2],\n",
       " [0, 0, 0])"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dl_iters, dl_idx, finished"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "gradient_accumulation = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'DistilBertTokenizer' object has no attribute 'max_len'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-62-ce758bbd5c3b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgradient_accumulation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m             \u001b[0mbatches\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdomain_dl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0mfinished\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/torch_B/lib/python3.6/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    343\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    344\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__next__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 345\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    346\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    347\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/torch_B/lib/python3.6/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    383\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    384\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 385\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    386\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    387\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/torch_B/lib/python3.6/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/torch_B/lib/python3.6/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/torch_B/lib/python3.6/site-packages/torch/utils/data/dataset.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, idx)\u001b[0m\n\u001b[1;32m    255\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    256\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 257\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindices\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    259\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/xformer-multi-source-domain-adaptation/datareader.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, item)\u001b[0m\n\u001b[1;32m    150\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTuple\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    151\u001b[0m         \u001b[0mrow\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 152\u001b[0;31m         \u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtext_to_batch_transformer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    153\u001b[0m         \u001b[0mlabel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m         \u001b[0mdomain\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/xformer-multi-source-domain-adaptation/datareader.py\u001b[0m in \u001b[0;36mtext_to_batch_transformer\u001b[0;34m(text, tokenizer, text_pair)\u001b[0m\n\u001b[1;32m     55\u001b[0m     \"\"\"\n\u001b[1;32m     56\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtext_pair\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m         \u001b[0minput_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0madd_special_tokens\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_length\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_len\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     58\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m         \u001b[0minput_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext_pair\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0madd_special_tokens\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_length\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_len\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext_pair\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/xformer-multi-source-domain-adaptation/datareader.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     55\u001b[0m     \"\"\"\n\u001b[1;32m     56\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtext_pair\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m         \u001b[0minput_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0madd_special_tokens\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_length\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_len\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     58\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m         \u001b[0minput_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext_pair\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0madd_special_tokens\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_length\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_len\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext_pair\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'DistilBertTokenizer' object has no attribute 'max_len'"
     ]
    }
   ],
   "source": [
    "for d in dl_idx:\n",
    "    domain_dl = dl_iters[d]\n",
    "    batches = []\n",
    "    try:\n",
    "        for j in range(gradient_accumulation):\n",
    "            batches.append(next(domain_dl))\n",
    "    except StopIteration:\n",
    "        finished[d] = 1\n",
    "        if len(batches) == 0:\n",
    "            continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PreTrainedTokenizer(name_or_path='distilbert-base-uncased', vocab_size=30522, model_max_len=512, is_fast=False, padding_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "        bert_model = 'bert-base-uncased'\n",
    "        bert_config = BertConfig.from_pretrained(bert_model, num_labels=2)\n",
    "        b_tokenizer = BertTokenizer.from_pretrained(bert_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PreTrainedTokenizer(name_or_path='bert-base-uncased', vocab_size=30522, model_max_len=512, is_fast=False, padding_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b_tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
